# -*- coding: utf-8 -*-
"""
Created on Sun Apr 12 17:49:15 2020

@author: Yhkun
"""
from bs4 import BeautifulSoup
import requests
from pyquery import PyQuery as pq
from urllib.parse import urlparse
import time

print("""
 __     ___     _                 _____                __     _______ _   _ ________          _______ 
 \ \   / / |   | |               / ____|               \ \   / / ____| \ | |  ____\ \        / / ____|
  \ \_/ /| |__ | | ___   _ _ __ | |     ___  _ __ ___   \ \_/ / |    |  \| | |__   \ \  /\  / / (___  
   \   / | '_ \| |/ / | | | '_ \| |    / _ \| '__/ _ \   \   /| |    | . ` |  __|   \ \/  \/ / \___ \ 
    | |  | | | |   <| |_| | | | | |___| (_) | | |  __/    | | | |____| |\  | |____   \  /\  /  ____) |
    |_|  |_| |_|_|\_\\__,_|_| |_|\_____\___/|_|  \___|    |_|  \_____|_| \_|______|   \/  \/  |_____/ 
                                                                                                      
                                                                                                            
""")


def getycwb():
    html = requests.get("http://ep.ycwb.com/epaper/ycwb/paperindex.htm").text
    soup = BeautifulSoup(html,'lxml')
    URL = soup.find(attrs={"http-equiv": "REFRESH"})['content'][7:]
    URL = "http://ep.ycwb.com/epaper/ycwb/"+URL
    html = requests.get(URL)
    html=html.content.decode('utf-8')
    #初始化为PyQuery对象
    doc = pq(html)
    url = doc('a').map(lambda i, e: pq(e)('a').attr('href'))
    
    weblist = []
    parse = urlparse(URL)
    
    for x in url:
        if x[:7] == 'content':
            weblist.append("http://ep.ycwb.com"+parse.path[:-10]+x)
    
            
        
    for k in weblist:
        time.sleep(0.5)
        html = requests.get(k)
        html = html.content.decode('utf-8')
        doc = pq(html)
        black = ['羊城晚报-A1','羊城晚报-导读']
        if doc.find("founder-content").text() == '' or doc('title').text() in black:
            pass
        else:
            doc.find("founder-content").text()
            doc('title').text()
            yield {"bt":doc('title').text(),"nr":doc.find("founder-content").text()}
            
    